package main

import (
	"bboying_spider/cos"
	"bboying_spider/data"
	"bboying_spider/db"
	"bboying_spider/utils"
	"encoding/json"
	"fmt"
	browser "github.com/EDDYCJY/fake-useragent"
	"github.com/gocolly/colly"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	"net/http"
	"os"
	"strings"
	"time"
)

var sugarLogger *zap.SugaredLogger

func InitLogger() {
	writeSyncer := getLogWriter()
	encoder := getEncoder()
	core := zapcore.NewCore(encoder, writeSyncer, zapcore.DebugLevel)

	logger := zap.New(core)
	sugarLogger = logger.Sugar()
}

func getEncoder() zapcore.Encoder {
	return zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig())
}

func getLogWriter() zapcore.WriteSyncer {
	//如果想要追加写入可以查看我的博客文件操作那一章
	file, _ := os.Create("./record-bboy.log")
	return zapcore.AddSync(file)
}

func main() {
	InitLogger()

	//bboy
	setKey := utils.BuildSpiderBboysSetKey()
	visitUrlFormat := "https://bgirlbboy.com/category/bboys/page/%d/"
	index := 1
	maxPage := 15
	visitUrl := fmt.Sprintf(visitUrlFormat, index)

	////bgirl
	//setKey := utils.BuildSpiderBgirlsSetKey()
	//visitUrlFormat := "https://bgirlbboy.com/category/bgirls/page/%d/"
	//index := 1
	//maxPage := 8
	//visitUrl := fmt.Sprintf(visitUrlFormat, index)

	defer sugarLogger.Sync()
	c := colly.NewCollector(
		colly.AllowURLRevisit(),
	)

	c.OnRequest(func(r *colly.Request) {
		//r.Headers.Set("User-Agent", RandomString())
		random := browser.Random()
		r.Headers.Set("User-Agent", random)
		//r.Headers.Set("User-Agent", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Mobile Safari/537.36 Edg/128.0.0.0")
	})

	c.OnHTML("main#main", func(main *colly.HTMLElement) {
		main.ForEach("article[id^='post']", func(i int, article *colly.HTMLElement) {
			/*
				获取图片信息
			*/
			figure := article.DOM.Find("figure.post-thumbnail")
			a := figure.Find("a[href]")
			imgPath, exist := a.Find("img[data-lazy-src]").Attr("data-lazy-src")
			if exist {
				//如果存在图片路径，则认为抓取正常
				//fmt.Println(imgPath)
				/*
					获取文字信息
				*/
				header := article.DOM.Find("header.entry-header")
				//解析名字
				nameA := header.Find("h2.entry-title").Find("a[href]")
				nickname := nameA.Text()
				detailUrl, exist := nameA.Attr("href")
				//详情跳转页必须存在
				if !exist {
					sugarLogger.Errorf("Error Page %d details not exist. nickname= %s", index, nickname)
				} else {
					//解析时间
					div := header.Find("div.entry-meta")
					timeDom := div.Find("time.entry-date.published")
					dateTimeStr, _ := timeDom.Attr("datetime")
					publishTime := time.Now()
					timeFormat := "2006-01-02T15:04:05+00:00"
					publishTime, err := time.Parse(timeFormat, dateTimeStr)
					if err != nil {
						publishTime = time.Now()
					}

					//fmt.Printf("%s url: %s publish: %s img: %s\n", nickname, detailUrl, publishTime, imgPath)
					id := utils.BuildDancerId(nickname, publishTime)

					// 发起 HTTP 请求下载文件
					resp, err := http.Get(imgPath)
					if err != nil {
						fmt.Println("Err 下载图片失败")
						sugarLogger.Errorf("Error Page %d download %s img : %s", index, nickname, err)
					} else {
						//获取正常
						fmt.Println("下载图片开始")
						key := fmt.Sprint("spider/", fmt.Sprintf("%s.jpg", id))
						if strings.HasSuffix(imgPath, ".jpg") {
							key = fmt.Sprint("spider/", fmt.Sprintf("%s.jpg", id))
						} else if strings.HasSuffix(imgPath, ".png") {
							key = fmt.Sprint("spider/", fmt.Sprintf("%s.png", id))
						} else if strings.HasSuffix(imgPath, ".webp") {
							key = fmt.Sprint("spider/", fmt.Sprintf("%s.webp", id))
						}
						downloadUrl, err := cos.Instance().PutFile(key, resp.Body)
						defer resp.Body.Close()
						if err == nil {
							imgPath = downloadUrl
						}

						dancer := data.Dancer{
							Id:          id,
							Nickname:    nickname,
							ImgUrl:      imgPath,
							PublishTime: publishTime.UnixMilli(),
							DetailUrl:   detailUrl,
							ClickCount:  0,
						}
						//log.Println("开启保存事务")
						db.Instance().StartPipeline()
						//将Mark解析为bytes
						jsonBytes, err := json.Marshal(dancer)
						if err != nil {
							sugarLogger.Errorf("Error Page %d parse json : %s", index, err)
						} else { //保存到DB
							err = db.Instance().SetValue(dancer.Id, string(jsonBytes))
							db.Instance().ZSet(setKey, dancer.Id, float64(time.Now().UnixNano()))
							err = db.Instance().Exec()
							if err != nil {
								fmt.Println("保存失败")
								sugarLogger.Errorf("Error Page %d save %s to db : %s", index, dancer.Nickname, err)
							} else {
								fmt.Println("保存成功：", dancer)
								sugarLogger.Infof("Info Page %d Save %s to db, Id is %s", index, dancer.Nickname, dancer.Id)
							}
						}
					}
				}

			}
		})
		index += 1
		if index > maxPage {
			return
		}
		visitUrl = fmt.Sprintf(visitUrlFormat, index)
		fmt.Println("访问：" + visitUrl)
		c.Visit(visitUrl)
	})

	fmt.Println("访问：" + visitUrl)
	c.Visit(visitUrl)
}
